[アップデート]Apache Iceberg形式のテーブルデータに最適化されたストレージAmazon S3 Tablesが発表されました

spark-shell \
 --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,software.amazon.s3tables:s3-tables-catalog-for-iceberg-runtime:0.1.3,software.amazon.awssdk:s3tables:2.29.26,software.amazon.awssdk:s3:2.29.26,software.amazon.awssdk:sts:2.29.26,software.amazon.awssdk:kms:2.29.26,software.amazon.awssdk:dynamodb:2.29.26,software.amazon.awssdk:kms:2.29.26,software.amazon.awssdk:glue:2.29.26 \
 --conf spark.sql.catalog.s3tablesbucket=org.apache.iceberg.spark.SparkCatalog \
 --conf spark.sql.catalog.s3tablesbucket.catalog-impl=software.amazon.s3tables.iceberg.S3TablesCatalog \
 --conf spark.sql.catalog.s3tablesbucket.warehouse=<Table bucketのARN> \
 --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
 --conf spark.driver.extraJavaOptions="-Djava.security.manager=allow"

spark.sql("create namespace if not exists s3tablesbucket.<任意のNamespace名>")

scala> spark.sql("SHOW NAMESPACES IN s3tablescatalog").show()
+---------------+
|      namespace|
+---------------+
|niino_namespace|
+---------------+

spark.sql("create table if not exists s3tablesbucket.niino_namespace.sales ( product string, amount int, timestamp timestamp) using iceberg ")

spark.sql("""
     INSERT INTO s3tablesbucket.niino_namespace.sales
     VALUES 
       ('Laptop', 1200, timestamp '2023-12-01 10:30:00'),
       ('Smartphone', 800, timestamp '2023-12-01 11:15:00'),
       ('Headphones', 150, timestamp '2023-12-02 09:45:00'),
       ('Monitor', 350, timestamp '2023-12-02 14:20:00'),
       ('Keyboard', 80, timestamp '2023-12-03 16:30:00'),
       ('Mouse', 45, timestamp '2023-12-03 16:35:00'),
       ('Tablet', 500, timestamp '2023-12-04 13:15:00'),
       ('Printer', 250, timestamp '2023-12-04 15:45:00'),
       ('Speaker', 120, timestamp '2023-12-05 10:20:00'),
       ('Webcam', 90, timestamp '2023-12-05 11:30:00')
     """)

scala> spark.sql("select * from s3tablesbucket.niino_namespace.sales").show()
+----------+------+-------------------+                                         
|   product|amount|          timestamp|
+----------+------+-------------------+
|    Laptop|  1200|2023-12-01 10:30:00|
|Smartphone|   800|2023-12-01 11:15:00|
|Headphones|   150|2023-12-02 09:45:00|
|   Monitor|   350|2023-12-02 14:20:00|
|  Keyboard|    80|2023-12-03 16:30:00|
（中略）
+----------+------+-------------------+

$  aws s3tables get-table-bucket-maintenance-configuration --table-bucket-arn="<Table bucketのARN>" 
{
    "tableBucketARN": "<テーブルのARN>",
    "configuration": {
        "icebergUnreferencedFileRemoval": {
            "status": "enabled",
            "settings": {
                "icebergUnreferencedFileRemoval": {
                    "unreferencedDays": 3,
                    "nonCurrentDays": 10
                }
            }
        }
    }
}

$ aws s3tables get-table-maintenance-configuration \
>    --table-bucket-arn="<Table bucketのARN>" \
>    --namespace="<Namespace名>" \
>    --name="<テーブル名>" 
{
    "tableARN": "<テーブルのARN>",
    "configuration": {
        "icebergCompaction": {
            "status": "enabled",
            "settings": {
                "icebergCompaction": {
                    "targetFileSizeMB": 512
                }
            }
        },
        "icebergSnapshotManagement": {
            "status": "enabled",
            "settings": {
                "icebergSnapshotManagement": {
                    "minSnapshotsToKeep": 1,
                    "maxSnapshotAgeHours": 120
                }
            }
        }
    }
}

sudo yum install java-devel

wget https://dlcdn.apache.org/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz

tar xvf spark-3.5.3-bin-hadoop3.tgz

sudo mv spark-3.5.3-bin-hadoop3 <任意のディレクトリ>

nano ~/.bashrc

export SPARK_HOME=<任意のディレクトリ>
export PATH=$PATH:$SPARK_HOME/bin

source ~/.bashrc

$ spark-shell --version
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 3.5.3
      /_/

Using Scala version 2.12.18, OpenJDK 64-Bit Server VM, 23.0.1
Branch HEAD
Compiled by user haejoon.lee on 2024-09-09T05:20:05Z
Revision 32232e9ed33bb16b93ad58cfde8b82e0f07c0970
Url https://github.com/apache/spark
Type --help for more information.